{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected = [\n",
    "    '/home/husein/ssd3/instructions/synthetic-codealpaca-v1-chatgpt4.jsonl',\n",
    "    '/home/husein/ssd3/instructions/synthetic-glaive_coder_raw_text.jsonl',\n",
    "    '/home/husein/ssd3/instructions/synthetic-oss_instruct-decontaminated.jsonl'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected_words = [\n",
    "    'kebutuhan',\n",
    "    'berbeda',\n",
    "    'bahwa',\n",
    "    'nomor',\n",
    "    'RMXX,XXX',\n",
    "    'kompleksitas',\n",
    "    'listrik',\n",
    "    'jawaban',\n",
    "    'teknis',\n",
    "    'berkualitas',\n",
    "    'mencoba',\n",
    "    'kampanye',\n",
    "    'komunitas',\n",
    "    'stabilitas',\n",
    "    'Stabilitas',\n",
    "    'metode',\n",
    "    'pria',\n",
    "    'butuh',\n",
    "    'jadwal',\n",
    "    'kasus',\n",
    "    'otomatis',\n",
    "    'populer',\n",
    "    'bisnis',\n",
    "    'probabilitas',\n",
    "    'rusak',\n",
    "    'kapasitas',\n",
    "    'rutinitas',\n",
    "    'pertama-tama',\n",
    "    'bagian',\n",
    "    'fungsionalitas',\n",
    "]\n",
    "\n",
    "indons = []\n",
    "def found_word(s, words):\n",
    "    global indons\n",
    "    for i in range(len(words)):\n",
    "        if words[i] in s:\n",
    "            indons.append(s)\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_code(s):\n",
    "    s = s.replace(' Kode ', ' Kod ')\n",
    "    s = s.replace(' kode ', ' kod ')\n",
    "    s = s.replace('Kode ', 'Kod ')\n",
    "    s = s.replace('kodenya', 'kodnya')\n",
    "    s = s.replace('Kodenya', 'Kodnya')\n",
    "    s = s.replace('opkode', 'opkod')\n",
    "    s = s.replace(' kode', ' kod')\n",
    "    s = s.replace('dikodekan', 'dikodkan')\n",
    "    s = s.replace('kodks', 'kodeks')\n",
    "    s = s.replace('kode ', 'kod ')\n",
    "    s = s.replace(' Kode', ' Kod')\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_news = glob('/home/husein/ssd3/google-translate-english-news/*.requested')\n",
    "len(english_news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "english_texts = glob('/home/husein/ssd3/google-translate-english-texts/*.requested')\n",
    "len(english_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "malay_news = glob('/home/husein/ssd3/google-translate-malay-news/*.requested')\n",
    "len(malay_news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hansard = glob('/home/husein/ssd3/google-translate-malaysia-hansard/*.requested')\n",
    "len(hansard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ens = [english_news, english_texts]\n",
    "mss = [malay_news, hansard]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reject(k):\n",
    "    if 'return JSON' in k:\n",
    "        return\n",
    "    if 'AI language model' in k:\n",
    "        return\n",
    "    if 'model bahasa AI' in k:\n",
    "        return\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'w') as fopen_jsonl:\n",
    "    \n",
    "    for f in selected:\n",
    "        with open(f) as fopen:\n",
    "            for l in fopen:\n",
    "                l = json.loads(l)\n",
    "                en = l['instruction_en']\n",
    "                ms = l['instruction_ms']\n",
    "\n",
    "                if not en or not ms:\n",
    "                    continue\n",
    "\n",
    "                if not reject(en) or not reject(ms):\n",
    "                    continue\n",
    "                \n",
    "                ms = replace_code(ms)\n",
    "        \n",
    "                if found_word(ms, rejected_words):\n",
    "                    continue\n",
    "                \n",
    "                if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                    d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('/home/husein/ssd3/translation/negeri-kelantan.jsonl') as fopen:\n",
    "        for l in fopen:\n",
    "            l = json.loads(l)\n",
    "            left = l['r']['translation']\n",
    "            right = l['original']['ms']\n",
    "            \n",
    "            if len(left) and len(left.split()) < 1200 and len(right) and len(right.split()) < 1200:\n",
    "\n",
    "                d = {\"translation\": {\"src\": left, \"tgt\": right, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            \n",
    "    with open('/home/husein/ssd3/translation/negeri-utara.jsonl') as fopen:\n",
    "        for l in fopen:\n",
    "            l = json.loads(l)\n",
    "            left = l['r']['translation']\n",
    "            right = l['original']['ms']\n",
    "            \n",
    "            if len(left) and len(left.split()) < 1200 and len(right) and len(right.split()) < 1200:\n",
    "\n",
    "                d = {\"translation\": {\"src\": left, \"tgt\": right, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('/home/husein/ssd3/gpt4all-1.3/translated-gpt4all-filtered-noncode.jsonl') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            l = json.loads(l)\n",
    "            try:\n",
    "                en = l['src']['prompt']\n",
    "                ms = l['r'][0]['result']\n",
    "                \n",
    "                if not reject(en) or not reject(ms):\n",
    "                    continue\n",
    "                    \n",
    "                ms = replace_code(ms)\n",
    "        \n",
    "                if found_word(ms, rejected_words):\n",
    "                    continue\n",
    "\n",
    "                if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                    d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "            except:\n",
    "                pass\n",
    "            \n",
    "            try:\n",
    "                en = l['src']['response']\n",
    "                ms = l['r'][1]['result']\n",
    "                \n",
    "                if not reject(en) or not reject(ms):\n",
    "                    continue\n",
    "                    \n",
    "                ms = replace_code(ms)\n",
    "        \n",
    "                if found_word(ms, rejected_words):\n",
    "                    continue\n",
    "                \n",
    "                if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                    d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            \n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!wc -l train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls -lh /home/husein/ssd3/translation/process-manglish.jsonl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('/home/husein/ssd3/translation/process-manglish.jsonl') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            l = json.loads(l)\n",
    "            left = l['left']\n",
    "            en = l['en']\n",
    "            ms = l['ms']\n",
    "            \n",
    "            if not reject(en) or not reject(ms):\n",
    "                continue\n",
    "\n",
    "            if len(left) and len(left.split()) < 1200 and len(en) and len(en.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": left, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                d = {\"translation\": {\"src\": en, \"tgt\": left, 'prefix': 'terjemah ke Manglish: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "            if len(left) and len(left.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": left, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                d = {\"translation\": {\"src\": ms, \"tgt\": left, 'prefix': 'terjemah ke Manglish: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "            if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    for f in [\n",
    "        '/home/husein/ssd3/translation/processed-ms-id.jsonl', \n",
    "        '/home/husein/ssd3/translation/processed-ms-jw.jsonl'\n",
    "    ]:\n",
    "        with open(f) as fopen:\n",
    "            for l in tqdm(fopen):\n",
    "                l = json.loads(l)\n",
    "                left = l['left']\n",
    "                right = l['right']\n",
    "                if len(left) and len(left.split()) < 1200 and len(right) and len(right.split()) < 1200:\n",
    "                    d = {\"translation\": {\"src\": right, \"tgt\": left, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('/home/husein/ssd3/translation/process-bjn-Latn.jsonl') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            l = json.loads(l)\n",
    "            left = l['left']\n",
    "            chatgpt = l['chatgpt']\n",
    "            google = l['google']\n",
    "            \n",
    "            if not reject(chatgpt) or not reject(google):\n",
    "                continue\n",
    "            \n",
    "            if len(left) and len(left.split()) < 1200 and len(google) and len(google.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": left, \"tgt\": google, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "            if len(chatgpt) and len(chatgpt.split()) < 1200 and len(google) and len(google.split()) < 1200:\n",
    "                d = {\"translation\": {\"src\": chatgpt, \"tgt\": google, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wc -l train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    for f in [\n",
    "        '/home/husein/ssd3/translation/processed-twitter.jsonl', \n",
    "        '/home/husein/ssd3/translation/processed-facebook.jsonl', \n",
    "        '/home/husein/ssd3/translation/processed-iium-confession.jsonl',\n",
    "        '/home/husein/ssd3/translation/processed-b.cari.com.my.jsonl',\n",
    "    ]:\n",
    "        with open(f) as fopen:\n",
    "            for l in tqdm(fopen):\n",
    "                try:\n",
    "                    l = json.loads(l)\n",
    "                    left = l['left']\n",
    "                    en = l['en']\n",
    "                    ms = l['ms']\n",
    "                    \n",
    "                    if not reject(en) or not reject(ms):\n",
    "                        continue\n",
    "\n",
    "                    if len(left) and len(left.split()) < 1200 and len(en) and len(en.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": left, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": left, 'prefix': 'terjemah ke pasar Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    if len(left) and len(left.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": left, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": left, 'prefix': 'terjemah ke pasar Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                    if len(ms) and len(ms.split()) < 1200 and len(en) and len(en.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "\n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "                except Exception as e:\n",
    "                    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wc -l train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    with open('gpt4all-code.jsonl') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            try:\n",
    "                l = json.loads(l)\n",
    "                left = l['prompt'][0]\n",
    "                right = l['prompt'][1]\n",
    "                \n",
    "                right = replace_code(right)\n",
    "        \n",
    "                if found_word(right, rejected_words):\n",
    "                    continue\n",
    "                \n",
    "                if len(left) and len(right):\n",
    "                    d = {\"translation\": {\"src\": left, \"tgt\": right, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": right, \"tgt\": left, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                \n",
    "                left = l['response'][0]\n",
    "                right = l['response'][1]\n",
    "                \n",
    "                right = replace_code(right)\n",
    "        \n",
    "                if found_word(right, rejected_words):\n",
    "                    continue\n",
    "                \n",
    "                if len(left) and len(right):\n",
    "                    d = {\"translation\": {\"src\": left, \"tgt\": right, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                    \n",
    "                    d = {\"translation\": {\"src\": right, \"tgt\": left, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                    fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "            except Exception as e:\n",
    "                print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wc -l train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "with open('train.json', 'a') as fopen_jsonl:\n",
    "    for fs in ens:\n",
    "        for f in fs:\n",
    "            with open(f) as fopen:\n",
    "                for l in tqdm(fopen):\n",
    "                    \n",
    "                    if random.random() > 0.4:\n",
    "                        continue\n",
    "                        \n",
    "                    data = json.loads(l)\n",
    "                    en = data['src']\n",
    "                    ms = data['r']['result']\n",
    "\n",
    "                    if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "    for fs in mss:\n",
    "        for f in fs:\n",
    "            with open(f) as fopen:\n",
    "                for l in tqdm(fopen):\n",
    "                    \n",
    "                    if 'malaysia-hansard' not in f and random.random() > 0.6:\n",
    "                        continue\n",
    "                    \n",
    "                    data = json.loads(l)\n",
    "                    if isinstance(data['src'], dict):\n",
    "                        s = data['src']['cleaned']\n",
    "                    else:\n",
    "                        s = data['src']\n",
    "                    ms = s\n",
    "                    en = data['r']['result']\n",
    "\n",
    "                    if len(en) and len(en.split()) < 1200 and len(ms) and len(ms.split()) < 1200:\n",
    "                        \n",
    "                        d = {\"translation\": {\"src\": en, \"tgt\": ms, 'prefix': 'terjemah ke Melayu: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')\n",
    "                        \n",
    "                        d = {\"translation\": {\"src\": ms, \"tgt\": en, 'prefix': 'terjemah ke Inggeris: '}}\n",
    "                        fopen_jsonl.write(f'{json.dumps(d)}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wc -l train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def found_word(s, words):\n",
    "    for i in range(len(words)):\n",
    "        if words[i] in s:\n",
    "            return words[i]\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!shuf train.json > shuffled-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!head -n 1 shuffled-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3.8 install mosaicml-streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from streaming import MDSWriter, LocalDataset\n",
    "\n",
    "columns = {\n",
    "    'src': 'str',\n",
    "    'tgt': 'str',\n",
    "    'prefix': 'str',\n",
    "}\n",
    "hashes = 'sha1', 'xxh64'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf mosaic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with MDSWriter(out='mosaic', columns=columns, compression=None, hashes=hashes) as out:\n",
    "    with open('shuffled-train.json') as fopen:\n",
    "        for l in tqdm(fopen):\n",
    "            l = json.loads(l)['translation']\n",
    "            out.write(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = LocalDataset('mosaic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(indons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in indons:\n",
    "    if 'pertama-tama' in i:\n",
    "        print(i)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rm -rf train.json shuffled-train.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!du -hs mosaic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp mosaic/* ~/ssd3/mosaic-noisy-translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ls -lh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf mosaic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
